# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

'''
    进行数据存储的地方
'''
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
from twisted.enterprise import adbapi
import codecs
import json
import MySQLdb
import MySQLdb.cursors


class KkspiderPipeline(object):
    def process_item(self, item, spider):
        return item


'''
    定制自己的图片pipeline，需要继承scrapy提供的image pipeline
    可以进行过滤（图片大小等）、
'''


class ArticleImagePipeline(ImagesPipeline):
    def item_completed(self, results, item, info):
        if "from_image_url" in item:
            for ok, value in results:
                image_file_path = value["path"]
                item["from_image_path"] = image_file_path
            return item

    pass


class JsonWithEncodingPipeline(object):
    # 自定义的json导出
    def __init__(self):
        self.file = codecs.open('article.json', 'w', encoding="utf-8")

    def process_item(self, item, spider):
        # 将item转换为json的形式，注意ensure_ascii=false，不然中文会出问题
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_close(self, spider):
        self.file.close()


class JsonExporterPipeline(object):
    # 调用scrapy提供的jsonExporter导出
    def __init__(self):
        self.file = open('articleExceptor.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding="utf-8", ensure_ascii=False)
        self.exporter.start_exporting()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

    def close(self, spider):
        self.exporter.finish_exporting()
        self.file.close()

# 使用数据库连接池的异步mysql插入操作
class MysqlTwistedPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    # 从配置中读取配置，穿进来一个settings对象
    @classmethod
    def from_settings(cls, settings):
        # 写法固定，详细参照connections的__init__
        dbparms = dict(
            host=settings["MYSQL_HOST"],
            user=settings["MYSQL_USERNAME"],
            password=settings["MYSQL_PASSWORD"],
            db=settings["MYSQL_DBNAME"],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True
        )
        # 可变传参,MySQLdb是固定写法
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        return cls(dbpool)

    # 使用twisted将insert变成异步
    def process_item(self, item, spider):
        query = self.dbpool.runInteraction(self.insert_item, item)
        # 处理异常
        query.addErrback(self.handle_error, item, spider)

    def insert_item(self, cursor, item):
        # 根据不同的item选择不同的insert
        insert_sql, params = item.get_sql()
        cursor.execute(insert_sql, params)

    # 处理异步插入的异常
    def handle_error(self, failure, item, spider):
        print("插入sql错误{}".format(failure))

